In [81]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt

from sklearn.manifold import TSNE
from sklearn.discriminant_analysis import LinearDiscriminantAnalysis
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.ensemble import BaggingClassifier
from sklearn.ensemble import AdaBoostClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import VotingClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.linear_model import LogisticRegression

from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import confusion_matrix
In [2]:
data = pd.read_csv('bank.csv')
In [3]:
data.head()
Out[3]:
age job marital education default balance housing loan contact day month duration campaign pdays previous poutcome y
0 30 unemployed married primary no 1787 no no cellular 19 oct 79 1 -1 0 unknown no
1 33 services married secondary no 4789 yes yes cellular 11 may 220 1 339 4 failure no
2 35 management single tertiary no 1350 yes no cellular 16 apr 185 1 330 1 failure no
3 30 management married tertiary no 1476 yes yes unknown 3 jun 199 4 -1 0 unknown no
4 59 blue-collar married secondary no 0 yes no unknown 5 may 226 1 -1 0 unknown no
In [4]:
preprocessed_data = data.replace({
    'jan': 1,
    'feb': 2,
    'mar': 3,
    'apr': 4,
    'may': 5,
    'jun': 6,
    'jul': 7,
    'aug': 8,
    'sep': 9,
    'oct': 10,
    'nov': 11,
    'dec': 12
})
data_dummies = pd.get_dummies(preprocessed_data)
data_dummies.head()
Out[4]:
age balance day month duration campaign pdays previous job_admin. job_blue-collar ... loan_yes contact_cellular contact_telephone contact_unknown poutcome_failure poutcome_other poutcome_success poutcome_unknown y_no y_yes
0 30 1787 19 10 79 1 -1 0 0 0 ... 0 1 0 0 0 0 0 1 1 0
1 33 4789 11 5 220 1 339 4 0 0 ... 1 1 0 0 1 0 0 0 1 0
2 35 1350 16 4 185 1 330 1 0 0 ... 0 1 0 0 1 0 0 0 1 0
3 30 1476 3 6 199 4 -1 0 0 0 ... 1 0 0 1 0 0 0 1 1 0
4 59 0 5 5 226 1 -1 0 0 1 ... 0 0 0 1 0 0 0 1 1 0

5 rows × 42 columns

In [5]:
data_dummies.isnull().sum()
Out[5]:
age                    0
balance                0
day                    0
month                  0
duration               0
campaign               0
pdays                  0
previous               0
job_admin.             0
job_blue-collar        0
job_entrepreneur       0
job_housemaid          0
job_management         0
job_retired            0
job_self-employed      0
job_services           0
job_student            0
job_technician         0
job_unemployed         0
job_unknown            0
marital_divorced       0
marital_married        0
marital_single         0
education_primary      0
education_secondary    0
education_tertiary     0
education_unknown      0
default_no             0
default_yes            0
housing_no             0
housing_yes            0
loan_no                0
loan_yes               0
contact_cellular       0
contact_telephone      0
contact_unknown        0
poutcome_failure       0
poutcome_other         0
poutcome_success       0
poutcome_unknown       0
y_no                   0
y_yes                  0
dtype: int64
In [6]:
print(data_dummies.columns)

Index(['age', 'balance', 'day', 'month', 'duration', 'campaign', 'pdays',
       'previous', 'job_admin.', 'job_blue-collar', 'job_entrepreneur',
       'job_housemaid', 'job_management', 'job_retired', 'job_self-employed',
       'job_services', 'job_student', 'job_technician', 'job_unemployed',
       'job_unknown', 'marital_divorced', 'marital_married', 'marital_single',
       'education_primary', 'education_secondary', 'education_tertiary',
       'education_unknown', 'default_no', 'default_yes', 'housing_no',
       'housing_yes', 'loan_no', 'loan_yes', 'contact_cellular',
       'contact_telephone', 'contact_unknown', 'poutcome_failure',
       'poutcome_other', 'poutcome_success', 'poutcome_unknown', 'y_no',
       'y_yes'],
      dtype='object')
In [48]:
min_max_scaler = MinMaxScaler()
# columns = ['age', 'balance', 'education_primary', 'duration', 'education_secondary', 'education_tertiary', 'marital_married', 'marital_single', 'marital_divorced', 'default_yes', 'housing_yes', 'loan_yes', 'poutcome_failure', 'poutcome_other', 'poutcome_success']
columns = ['age','duration']
X = pd.DataFrame(data=min_max_scaler.fit_transform(data_dummies.loc[:, columns]), columns=columns)
X.head()
# y = data_dummies.y_yes
# y.head()
Out[48]:
age duration
0 0.161765 0.024826
1 0.205882 0.071500
2 0.235294 0.059914
3 0.161765 0.064548
4 0.588235 0.073486
In [32]:
plt.hist(y)
Out[32]:
(array([4000.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,    0.,
         521.]),
 array([0. , 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ]),
 <BarContainer object of 10 artists>)
In [33]:
plt.hist(X.balance)
Out[33]:
(array([4.111e+03, 3.400e+02, 4.700e+01, 1.700e+01, 4.000e+00, 0.000e+00,
        1.000e+00, 0.000e+00, 0.000e+00, 1.000e+00]),
 array([0. , 0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1. ]),
 <BarContainer object of 10 artists>)
In [50]:
tsne = TSNE()
tsne_transformed_data = tsne.fit_transform(X)

---------------------------------------------------------------------------
KeyboardInterrupt                         Traceback (most recent call last)
<ipython-input-50-825010eeadab> in <module>
      1 tsne = TSNE()
----> 2 tsne_transformed_data = tsne.fit_transform(X)

/usr/local/lib/python3.8/site-packages/sklearn/manifold/_t_sne.py in fit_transform(self, X, y)
    889             Embedding of the training data in low-dimensional space.
    890         """
--> 891         embedding = self._fit(X)
    892         self.embedding_ = embedding
    893         return self.embedding_

/usr/local/lib/python3.8/site-packages/sklearn/manifold/_t_sne.py in _fit(self, X, skip_num_points)
    798         degrees_of_freedom = max(self.n_components - 1, 1)
    799 
--> 800         return self._tsne(P, degrees_of_freedom, n_samples,
    801                           X_embedded=X_embedded,
    802                           neighbors=neighbors_nn,

/usr/local/lib/python3.8/site-packages/sklearn/manifold/_t_sne.py in _tsne(self, P, degrees_of_freedom, n_samples, X_embedded, neighbors, skip_num_points)
    839         # higher learning rate controlled via the early exaggeration parameter
    840         P *= self.early_exaggeration
--> 841         params, kl_divergence, it = _gradient_descent(obj_func, params,
    842                                                       **opt_args)
    843         if self.verbose:

/usr/local/lib/python3.8/site-packages/sklearn/manifold/_t_sne.py in _gradient_descent(objective, p0, it, n_iter, n_iter_check, n_iter_without_progress, momentum, learning_rate, min_gain, min_grad_norm, verbose, args, kwargs)
    357         kwargs['compute_error'] = check_convergence or i == n_iter - 1
    358 
--> 359         error, grad = objective(p, *args, **kwargs)
    360         grad_norm = linalg.norm(grad)
    361 

/usr/local/lib/python3.8/site-packages/sklearn/manifold/_t_sne.py in _kl_divergence_bh(params, P, degrees_of_freedom, n_samples, n_components, angle, skip_num_points, verbose, compute_error, num_threads)
    257 
    258     grad = np.zeros(X_embedded.shape, dtype=np.float32)
--> 259     error = _barnes_hut_tsne.gradient(val_P, X_embedded, neighbors, indptr,
    260                                       grad, angle, n_components, verbose,
    261                                       dof=degrees_of_freedom,

KeyboardInterrupt:
In [51]:
plt.scatter(X.iloc[:, [0]], X.iloc[:, [1]], c=y)
Out[51]:
<matplotlib.collections.PathCollection at 0x1212119a0>
In [49]:
plt.scatter(tsne_transformed_data[:, 0], tsne_transformed_data[:, 1], c=y)
Out[49]:
<matplotlib.collections.PathCollection at 0x125c0d670>
In [39]:
lda = LinearDiscriminantAnalysis()
lda_transformed_data = lda.fit_transform(X, y)
In [40]:
plt.scatter([0 for _ in range(lda_transformed_data.shape[0])], lda_transformed_data[:, 0], c=y)
Out[40]:
<matplotlib.collections.PathCollection at 0x125c51ac0>
In [14]:
X.head()
Out[14]:
age balance education_primary education_secondary education_tertiary marital_married marital_single marital_divorced default_yes housing_yes loan_yes poutcome_failure poutcome_other poutcome_success
0 30 1787 1 0 0 1 0 0 0 0 0 0 0 0
1 33 4789 0 1 0 1 0 0 0 1 1 1 0 0
2 35 1350 0 0 1 0 1 0 0 1 0 1 0 0
3 30 1476 0 0 1 1 0 0 0 1 1 0 0 0
4 59 0 0 1 0 1 0 0 0 1 0 0 0 0
In [52]:
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3)
In [97]:
knn = KNeighborsClassifier(n_neighbors=21, weights='distance')
knn.fit(X_train, y_train)
knn.score(X_test, y_test)
Out[97]:
0.8717759764185704
In [98]:
confusion_matrix(y_test, knn.predict(X_test))
Out[98]:
array([[1152,   51],
       [ 123,   31]])
In [53]:
tree = DecisionTreeClassifier(min_samples_split=20, max_depth=5)
tree.fit(X_train, y_train)
tree.score(X_test, y_test)
Out[53]:
0.8843036109064112
In [61]:
bc_tree = DecisionTreeClassifier(min_samples_split=10, max_depth=5)
bc = BaggingClassifier(bc_tree, n_estimators=100, max_samples=0.6)
bc.fit(X_train, y_train)
bc.score(X_test, y_test)
Out[61]:
0.8872512896094326
In [55]:
ab_tree = DecisionTreeClassifier(min_samples_split=20, max_depth=5)
ab = AdaBoostClassifier(ab_tree, n_estimators=50)
ab.fit(X_train, y_train)
ab.score(X_test, y_test)
Out[55]:
0.8835666912306559
In [56]:
ab_tree_2 = DecisionTreeClassifier(min_samples_split=20, max_depth=10, class_weight={0: 1, 1: 8})
ab_2 = AdaBoostClassifier(ab_tree_2, n_estimators=50)
abbc = BaggingClassifier(ab_2, n_estimators=100, max_samples=0.4)
abbc.fit(X_train, y_train)
abbc.score(X_test, y_test)
Out[56]:
0.8820928518791452
In [57]:
rf = RandomForestClassifier(n_estimators=50, min_samples_split=50, max_depth=5)
rf.fit(X_train, y_train)

list(zip(X.columns, rf.feature_importances_))
Out[57]:
[('age', 0.19601163375601824), ('duration', 0.8039883662439818)]
In [58]:
rf.score(X_test, y_test)
Out[58]:
0.8872512896094326
In [103]:
tree_v = DecisionTreeClassifier(min_samples_split=20, max_depth=5, class_weight={0: 1, 1: 2})
knn_v = KNeighborsClassifier(n_neighbors=21, weights='distance')
lr_v = LogisticRegression(class_weight={0: 1, 1: 2})
clf = VotingClassifier(estimators=[('tree', tree_v), ('knn', knn_v), ('lr', lr_v)], weights=[1,1,1])
clf.fit(X_train, y_train)

# for i in range(X_test.shape[0]):
#     x = X_test.iloc[[i], :]
#     y = y_test.values[i]
#     print(clf.estimators_[0].predict(x), clf.estimators_[1].predict(x), clf.estimators_[2].predict(x), clf.predict(x), y)

clf.score(X_test, y_test)
Out[103]:
0.8850405305821666
In [104]:
confusion_matrix(y_test, clf.predict(X_test))
Out[104]:
array([[1165,   38],
       [ 118,   36]])